import pandas as pd

df = pd.read_csv("/content/drive/MyDrive/ds/train.csv", encoding="utf-8")
facts = df['fact']
labels = df['label']

print(facts[0])
print(labels[0])

""" 犯人描述
罪犯吴智信，男...执行机关所提请减刑的建议，符合法律规定，本院根据该犯未履行原判财产刑的事实及其改造表现事实酌情予以减刑 
12 可减刑年份
"""
import re

new_facts = []

for fact in facts:
    fact = re.sub(r"[，。、；‘’]", " ", fact)
    new_facts.append(fact)

# 替换掉描述中的标点符号

print(new_facts[0])

"""
'罪犯陈中盛 男 1954年xx月xx日出生 汉族 湖南省永兴县人 文盲 现在河南省新郑监狱服刑 ... '
"""
from tensorflow.keras.preprocessing.text import Tokenizer

# 词库模型构建

model = Tokenizer()
model.fit_on_texts(new_facts)

word_index = model.word_index  # 返回的是字典

print(word_index)
"""
{
 '确有悔改表现': 1,
 '本院认为': 2,
 '现已审理终结': 3,
 '经审理查明': 4,
 '男': 5,
 '汉族': 6,
 '文化': 7,
 '接受教育改造': 8,
 '报送本院审理': 9,
 '努力完成劳动任务': 10,
 '本院依法组成合议庭进行了审理': 11,
 '接受教育改造；积极参加思想': 12,
 '初中文化': 13,
 '职业技术教育；积极参加劳动': 14,
 '积极参加思想': 15,
 '判决发生法律效力后交付执行': 16,
}
"""

print(len(word_index))  # 479374
from sklearn.model_selection import train_test_split
from tensorflow.keras.preprocessing.sequence import pad_sequences

# 划分训练和测试集
X_train, X_test, y_train, y_test = train_test_split(new_facts, labels)

# 将每个描述转换成矩阵
X_train_fit = model.texts_to_sequences(X_train)
X_test_fit = model.texts_to_sequences(X_test)

# 输出最大矩阵长度
i = []
for j in X_test_fit:
    i.append(len(j))

print(max(i))

# pad_sequences 填充 统一矩阵大小 padding="post" 表示在后方填充

X_train_padd = pad_sequences(X_train_fit, maxlen=250, padding="post")
X_test_padd = pad_sequences(X_test_fit, maxlen=250, padding="post")
import tensorflow as tf
from tensorflow.keras.layers import Dense, Embedding, GlobalAveragePooling1D, Dropout

model1 = tf.keras.Sequential([
    Embedding(500000, 64, input_length=250),  # 嵌入层 （词库大小，嵌入向量的维度，输入大小）
    GlobalAveragePooling1D(),  # 平均池化层 返回平均值
    Dense(32, activation="relu"),
    Dropout(0.2),
    Dense(1)
])
model1.summary()

model1.compile(optimizer="adam", loss="mse", metrics=['mean_absolute_error', 'accuracy'])

history = model1.fit(X_train_padd, y_train, batch_size=512, epochs=20, validation_data=(X_test_padd, y_test), verbose=1)

"""
Model: "sequential_2"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 embedding_3 (Embedding)     (None, 250, 64)           32000000  

 global_average_pooling1d_3  (None, 64)                0         
  (GlobalAveragePooling1D)                                       

 dense_4 (Dense)             (None, 32)                2080      

 dropout_2 (Dropout)         (None, 32)                0         

 dense_5 (Dense)             (None, 1)                 33        

=================================================================
Total params: 32002113 (122.08 MB)
Trainable params: 32002113 (122.08 MB)
Non-trainable params: 0 (0.00 Byte)
_________________________________________________________________
"""

"""
...
Epoch 100/100
74/74 - 1s - loss: 1.3636 - mean_absolute_error: 0.7654 - accuracy: 0.0255 - val_loss: 8.7319 - val_mean_absolute_error: 2.1841 - val_accuracy: 0.0136 - 1s/epoch - 20ms/step
"""